Chapter 15 - Serving Models Locally

This chapter wraps a GGUF model in a local HTTP service. The pattern is deliberately plain: start llama-server, wait until /v1/models responds, send requests, collect timings, and terminate the process.

from __future__ import annotations

import contextlib
import shlex
import socket
import subprocess
import time
from pathlib import Path

import pandas as pd
import requests

pd.set_option("display.max_colwidth", 120)

LLAMA_ROOT = Path.home() / "tmp" / "llama-cpp"
SERVER = LLAMA_ROOT / "llama.cpp" / "build" / "bin" / "llama-server"
MODEL = LLAMA_ROOT / "models" / "Qwen_Qwen3.5-4B-Q4_K_M.gguf"

FIXED_FLAGS = [
    ("-m", str(MODEL)),
    ("-c", "1024"),
    ("-np", "1"),
    ("--cache-type-k", "q8_0"),
    ("--cache-type-v", "q8_0"),
    ("--alias", "qwen-local"),
    ("--host", "127.0.0.1"),
]
SERVER_ATTEMPTS = [
    {"mode": "full_gpu", "gpu_layers": "all", "flash_attn": "on"},
    {"mode": "cpu_only", "gpu_layers": "0", "flash_attn": "on"},
]

flag_table = pd.concat([
    pd.DataFrame(FIXED_FLAGS, columns=["flag", "value"]),
    pd.DataFrame(SERVER_ATTEMPTS),
], ignore_index=True)
flag_table

	flag	value	mode	gpu_layers	flash_attn
0	-m	/home/alal/tmp/llama-cpp/models/Qwen_Qwen3.5-4B-Q4_K_M.gguf	NaN	NaN	NaN
1	-c	1024	NaN	NaN	NaN
2	-np	1	NaN	NaN	NaN
3	--cache-type-k	q8_0	NaN	NaN	NaN
4	--cache-type-v	q8_0	NaN	NaN	NaN
5	--alias	qwen-local	NaN	NaN	NaN
6	--host	127.0.0.1	NaN	NaN	NaN
7	NaN	NaN	full_gpu	all	on
8	NaN	NaN	cpu_only	0	on

def free_port() -> int:
    with socket.socket() as sock:
        sock.bind(("127.0.0.1", 0))
        return sock.getsockname()[1]

def server_command(port: int, gpu_layers: str = "all", flash_attn: str = "on") -> list[str]:
    command = [str(SERVER)]
    for flag, value in FIXED_FLAGS:
        command.extend([flag, value])
    command.extend([
        "-ngl", gpu_layers,
        "-fa", flash_attn,
        "--port", str(port),
        "--jinja",
        "--no-webui",
        "--log-disable",
    ])
    return command

@contextlib.contextmanager
def running_server():
    errors = []
    process = None
    try:
        for attempt in SERVER_ATTEMPTS:
            port = free_port()
            command = server_command(port, attempt["gpu_layers"], attempt["flash_attn"])
            process = subprocess.Popen(command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, text=True)
            base_url = f"http://127.0.0.1:{port}"
            deadline = time.time() + 90
            ready = False
            while time.time() < deadline:
                if process.poll() is not None:
                    errors.append(f"{attempt['mode']} exited with code {process.returncode}")
                    break
                try:
                    ready_response = requests.get(f"{base_url}/v1/models", timeout=2)
                    if ready_response.ok:
                        ready = True
                        break
                except requests.RequestException:
                    time.sleep(1)
            if ready:
                print(f"server_mode: {attempt['mode']}")
                yield base_url, command
                return
            if process.poll() is None:
                process.terminate()
                try:
                    process.wait(timeout=15)
                except subprocess.TimeoutExpired:
                    process.kill()
                    process.wait(timeout=15)
            process = None
        raise RuntimeError("server did not become ready; " + "; ".join(errors))
    finally:
        if process is not None and process.poll() is None:
            process.terminate()
            try:
                process.wait(timeout=15)
            except subprocess.TimeoutExpired:
                process.kill()
                process.wait(timeout=15)

example_command = server_command(8080)
print(shlex.join(example_command))

/home/alal/tmp/llama-cpp/llama.cpp/build/bin/llama-server -m /home/alal/tmp/llama-cpp/models/Qwen_Qwen3.5-4B-Q4_K_M.gguf -c 1024 -np 1 --cache-type-k q8_0 --cache-type-v q8_0 --alias qwen-local --host 127.0.0.1 -ngl all -fa on --port 8080 --jinja --no-webui --log-disable

with running_server() as (base_url, command):
    models = requests.get(f"{base_url}/v1/models", timeout=10).json()
    row = models["data"][0]
    print(f"Serving from {base_url}")
    print(f"Model id: {row.get('id')}")
    print(f"Object type: {row.get('object')}")
    print(f"Command uses {len(command)} command-line arguments.")

server_mode: cpu_only
Serving from http://127.0.0.1:45717
Model id: qwen-local
Object type: model
Command uses 24 command-line arguments.

def native_completion(base_url: str, question: str) -> dict:
    prompt = f"Q: {question}\nA:"
    response = requests.post(
        f"{base_url}/completion",
        json={
            "prompt": prompt,
            "n_predict": 72,
            "temperature": 0,
            "stop": ["\nQ:"],
        },
        timeout=90,
    )
    response.raise_for_status()
    payload = response.json()
    timings = payload.get("timings", {})
    return {
        "question": question,
        "answer": payload.get("content", "").strip(),
        "predicted_tokens": timings.get("predicted_n"),
        "predicted_tokens_per_s": timings.get("predicted_per_second"),
    }

questions = [
    "What is the role of llama-server in a local LLM application?",
    "Why bind a development server to 127.0.0.1?",
    "What should an application log for local model requests?",
]

with running_server() as (base_url, _):
    rows = [native_completion(base_url, question) for question in questions]

served = pd.DataFrame(rows)
served["predicted_tokens_per_s"] = served["predicted_tokens_per_s"].round(2)
served

server_mode: full_gpu

	question	answer	predicted_tokens	predicted_tokens_per_s
0	What is the role of llama-server in a local LLM application?	llama-server is a tool that provides a server-side interface for running LLMs locally. It allows users to interact w...	46	138.30
1	Why bind a development server to 127.0.0.1?	Because it's the only way to access the server from the local machine.	18	140.88
2	What should an application log for local model requests?	An application should log the following information for local model requests:\n1. Timestamp\n2. Model name\n3. Reque...	72	138.31

summary = served[["predicted_tokens", "predicted_tokens_per_s"]].describe().loc[["mean", "min", "max"]]
summary

	predicted_tokens	predicted_tokens_per_s
mean	45.333333	139.163333
min	18.000000	138.300000
max	72.000000	140.880000

For application code, the important boundary is the HTTP contract, not the model binary. Keep the server command, model path, context size, quantization, and timing fields with the run metadata so later evaluations can distinguish model quality from serving configuration.